#coding=utf-8
import os
import MySQLdb
import time
from biquge import BIQUGE


conn = MySQLdb.connect(host='127.0.0.1',user='root',passwd='root',db='xiaoshuo',port=3306,charset='utf8')
cur = conn.cursor()
BASE_DIR = os.path.dirname(__file__)

book = BIQUGE()

def makedir(book_id):
    ok = cur.execute("select read_url from web_titleurl where book_id='%s'" % book_id)

    if not ok:
        return False
    urls = cur.fetchall()
    
    path = os.path.join(BASE_DIR,str(book_id))
    if not os.path.isdir(path):
        os.makedirs(path)
    errorid = []
    for url in urls:
        filename = os.path.join(path,url[0])
        if os.path.isfile(filename):
            continue
        while True:
            try:
                real_url = 'http://www.biquge.la/book/%s/%s'%(book_id,url[0])
                content = book.getOnePage(real_url)
                break
            except Exception as e:
                print 'wait for a second...'
                if e.code == 404:
                    print 'None Content'
                    errorid.append(str(book_id)+'/'+url[0])
                    return False
                time.sleep(1)

        
        with open(filename,'w') as f:
            f.write(str(content))

    if errorid:
        with open('errorid.txt','w+') as f:
            for i in errorid:
                f.write(i+'\n')
            
        
for i in range(1,3729):
    print i
    makedir(i)
